{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ddae6771",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "train_dir = \"./data/ruijin_round1_train2_20181022\"\n",
    "\n",
    "\n",
    "def get_entities(data_path):\n",
    "    \"\"\"\n",
    "    获取要提取的实体的种类\n",
    "    :param data_path: 数据路径\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    entities = {}\n",
    "    for file_name in os.listdir(data_path):\n",
    "        if file_name.split(\".\")[1] == \"ann\":\n",
    "            with open(train_dir + \"/\" + file_name, mode=\"r\", encoding=\"utf8\") as f:\n",
    "                for line in f:\n",
    "                    entity_name = line.split(\"\\t\")[1].split(\" \")[0]\n",
    "                    if entity_name in entities:\n",
    "                        entities[entity_name] += 1\n",
    "                    else:\n",
    "                        entities[entity_name] = 1\n",
    "    return entities\n",
    "\n",
    "def get_entity_dict(entities):\n",
    "    \"\"\"\n",
    "    构建实体字典，实体对应下标\n",
    "    为了后面可以将标签转成下标数字\n",
    "\n",
    "    id2lable 根据id获取标签：\n",
    "    ['O', 'B-Disease', 'I-Disease', 'B-Test', 'I-Test',....]\n",
    "    lable2id 根据标签获取id：\n",
    "     {'O': 0, 'B-Disease': 1, 'I-Disease': 2, 'B-Test': 3, 'I-Test': 4, ...}\n",
    "\n",
    "    :param entities:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "\n",
    "    id2lable = []\n",
    "\n",
    "    id2lable.append('O')\n",
    "    for en in entities.keys():\n",
    "        id2lable.append(\"B-\" + en)\n",
    "        id2lable.append(\"I-\" + en)\n",
    "\n",
    "    lable2id = {id2lable[i] : i for i in range(len(id2lable))}\n",
    "    return id2lable,lable2id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "efca06d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "前\n",
      "2型糖尿病及其并发症已经成为危害公众\n",
      "健康的主要疾病之一,\n"
     ]
    }
   ],
   "source": [
    "# 切分文本句子，获取切分的索引\n",
    "import re\n",
    "parttern = '。|, |,|；|;|\\.'\n",
    "point = []\n",
    "with open ('data/ruijin_round1_train2_20181022/0.txt',encoding='utf8') as f:\n",
    "    text = f.read()\n",
    "    print(text[28])\n",
    "    print(text[30:59])\n",
    "    for item in re.finditer(parttern,text):\n",
    "        point.append(item.span()[0])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f9c1267e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[29,\n",
       " 58,\n",
       " 86,\n",
       " 96,\n",
       " 112,\n",
       " 138,\n",
       " 156,\n",
       " 174,\n",
       " 198,\n",
       " 217,\n",
       " 263,\n",
       " 324,\n",
       " 340,\n",
       " 345,\n",
       " 350,\n",
       " 405,\n",
       " 498,\n",
       " 518,\n",
       " 521,\n",
       " 524,\n",
       " 547,\n",
       " 567,\n",
       " 579,\n",
       " 595,\n",
       " 619,\n",
       " 659,\n",
       " 680,\n",
       " 729,\n",
       " 741,\n",
       " 759,\n",
       " 782,\n",
       " 791,\n",
       " 826,\n",
       " 832,\n",
       " 950,\n",
       " 960,\n",
       " 983,\n",
       " 991,\n",
       " 996,\n",
       " 1018,\n",
       " 1101,\n",
       " 1111,\n",
       " 1122,\n",
       " 1126,\n",
       " 1146,\n",
       " 1182,\n",
       " 1189,\n",
       " 1224,\n",
       " 1234,\n",
       " 1257,\n",
       " 1281,\n",
       " 1301,\n",
       " 1309,\n",
       " 1332,\n",
       " 1358,\n",
       " 1363,\n",
       " 1395,\n",
       " 1397,\n",
       " 1407,\n",
       " 1415,\n",
       " 1427,\n",
       " 1460,\n",
       " 1471,\n",
       " 1477,\n",
       " 1480,\n",
       " 1511,\n",
       " 1522,\n",
       " 1526,\n",
       " 1542,\n",
       " 1550,\n",
       " 1575,\n",
       " 1600,\n",
       " 1618,\n",
       " 1628,\n",
       " 1639,\n",
       " 1642,\n",
       " 1645,\n",
       " 1661,\n",
       " 1695,\n",
       " 1722,\n",
       " 1728,\n",
       " 1738,\n",
       " 1744,\n",
       " 1759,\n",
       " 1788,\n",
       " 1807,\n",
       " 1815,\n",
       " 1821,\n",
       " 1841,\n",
       " 1844,\n",
       " 1852,\n",
       " 1859,\n",
       " 1874,\n",
       " 1889,\n",
       " 1917,\n",
       " 1933,\n",
       " 1956,\n",
       " 1961,\n",
       " 1966,\n",
       " 2009,\n",
       " 2023,\n",
       " 2026,\n",
       " 2062,\n",
       " 2084,\n",
       " 2109,\n",
       " 2112,\n",
       " 2141,\n",
       " 2144,\n",
       " 2162,\n",
       " 2178,\n",
       " 2191,\n",
       " 2199,\n",
       " 2231,\n",
       " 2234,\n",
       " 2256,\n",
       " 2278,\n",
       " 2282,\n",
       " 2306,\n",
       " 2339,\n",
       " 2345,\n",
       " 2361,\n",
       " 2374,\n",
       " 2404,\n",
       " 2414,\n",
       " 2441,\n",
       " 2444,\n",
       " 2466,\n",
       " 2469,\n",
       " 2487,\n",
       " 2495,\n",
       " 2498,\n",
       " 2521,\n",
       " 2533,\n",
       " 2558,\n",
       " 2564,\n",
       " 2575,\n",
       " 2585,\n",
       " 2617,\n",
       " 2627,\n",
       " 2630,\n",
       " 2644,\n",
       " 2667,\n",
       " 2684,\n",
       " 2702,\n",
       " 2711,\n",
       " 2734,\n",
       " 2742,\n",
       " 2768,\n",
       " 2786,\n",
       " 2811,\n",
       " 2826,\n",
       " 2839,\n",
       " 2842,\n",
       " 2845,\n",
       " 2867,\n",
       " 2875,\n",
       " 2896,\n",
       " 2911,\n",
       " 2922,\n",
       " 2949,\n",
       " 2966,\n",
       " 2994,\n",
       " 3030,\n",
       " 3036,\n",
       " 3053,\n",
       " 3066,\n",
       " 3097,\n",
       " 3112,\n",
       " 3122,\n",
       " 3162,\n",
       " 3165,\n",
       " 3169,\n",
       " 3188,\n",
       " 3205,\n",
       " 3216,\n",
       " 3225,\n",
       " 3231,\n",
       " 3248,\n",
       " 3251,\n",
       " 3258,\n",
       " 3270,\n",
       " 3275,\n",
       " 3302,\n",
       " 3341,\n",
       " 3345,\n",
       " 3372,\n",
       " 3396,\n",
       " 3423,\n",
       " 3433,\n",
       " 3452,\n",
       " 3495,\n",
       " 3532,\n",
       " 3540,\n",
       " 3566,\n",
       " 3591,\n",
       " 3598,\n",
       " 3634,\n",
       " 3647,\n",
       " 3656,\n",
       " 3678,\n",
       " 3691,\n",
       " 3699,\n",
       " 3703,\n",
       " 3735,\n",
       " 3763,\n",
       " 3775,\n",
       " 3805,\n",
       " 3839,\n",
       " 3860,\n",
       " 3871,\n",
       " 3895,\n",
       " 3912,\n",
       " 3934,\n",
       " 3954,\n",
       " 3973,\n",
       " 4002,\n",
       " 4009,\n",
       " 4019,\n",
       " 4034,\n",
       " 4060,\n",
       " 4071,\n",
       " 4090,\n",
       " 4093,\n",
       " 4120,\n",
       " 4132,\n",
       " 4160,\n",
       " 4230,\n",
       " 4243,\n",
       " 4248,\n",
       " 4263,\n",
       " 4273,\n",
       " 4292,\n",
       " 4295,\n",
       " 4298,\n",
       " 4302,\n",
       " 4340,\n",
       " 4370,\n",
       " 4378,\n",
       " 4388,\n",
       " 4397,\n",
       " 4406,\n",
       " 4417,\n",
       " 4421,\n",
       " 4432,\n",
       " 4465,\n",
       " 4479,\n",
       " 4574,\n",
       " 4577,\n",
       " 4585,\n",
       " 4588,\n",
       " 4591,\n",
       " 4614,\n",
       " 4615,\n",
       " 4618,\n",
       " 4625,\n",
       " 4645,\n",
       " 4666,\n",
       " 4710,\n",
       " 4733,\n",
       " 4793,\n",
       " 4806,\n",
       " 4838,\n",
       " 4852,\n",
       " 4871,\n",
       " 4874,\n",
       " 4877,\n",
       " 4910,\n",
       " 4931,\n",
       " 4944,\n",
       " 4969,\n",
       " 5004,\n",
       " 5018,\n",
       " 5022,\n",
       " 5042,\n",
       " 5051,\n",
       " 5061,\n",
       " 5074,\n",
       " 5089,\n",
       " 5098,\n",
       " 5106,\n",
       " 5122,\n",
       " 5134,\n",
       " 5141,\n",
       " 5152,\n",
       " 5171,\n",
       " 5181,\n",
       " 5192,\n",
       " 5204,\n",
       " 5213,\n",
       " 5238,\n",
       " 5268,\n",
       " 5286,\n",
       " 5299,\n",
       " 5307,\n",
       " 5314,\n",
       " 5323,\n",
       " 5331,\n",
       " 5340,\n",
       " 5354,\n",
       " 5382,\n",
       " 5396,\n",
       " 5404,\n",
       " 5412,\n",
       " 5444,\n",
       " 5474,\n",
       " 5513,\n",
       " 5525,\n",
       " 5557,\n",
       " 5560,\n",
       " 5581,\n",
       " 5605,\n",
       " 5623,\n",
       " 5627,\n",
       " 5647,\n",
       " 5672,\n",
       " 5687,\n",
       " 5726,\n",
       " 5751,\n",
       " 5756,\n",
       " 5775,\n",
       " 5787,\n",
       " 5806,\n",
       " 5828,\n",
       " 5834,\n",
       " 5854,\n",
       " 5879,\n",
       " 5885,\n",
       " 5891,\n",
       " 5897,\n",
       " 5903,\n",
       " 5909,\n",
       " 5929,\n",
       " 5949,\n",
       " 5959,\n",
       " 5968,\n",
       " 5997,\n",
       " 6032,\n",
       " 6037,\n",
       " 6044,\n",
       " 6051,\n",
       " 6062,\n",
       " 6102,\n",
       " 6131,\n",
       " 6140,\n",
       " 6163,\n",
       " 6184,\n",
       " 6218,\n",
       " 6236,\n",
       " 6269,\n",
       " 6280,\n",
       " 6296,\n",
       " 6319,\n",
       " 6327,\n",
       " 6357,\n",
       " 6374,\n",
       " 6387,\n",
       " 6394,\n",
       " 6412,\n",
       " 6442,\n",
       " 6450,\n",
       " 6470,\n",
       " 6515,\n",
       " 6526,\n",
       " 6535,\n",
       " 6572,\n",
       " 6607,\n",
       " 6772]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "point"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
